import re
import math
import requests

from lxml import etree


def page_request(url):
    html = requests.get(url=url, headers=headers).text
    return etree.HTML(html)


def page_get():
    html = page_request(base_url)
    # 获取分类列表
    type_list = html.xpath('//div[@class="types"]/span')
    for m_type in type_list:
        href = m_type.xpath('./a/@href')
        type_n = re.compile(r'type=(.*?)&').findall(href[0])
        # 请求对应的列表页
        json_url = f'https://movie.douban.com/j/chart/top_list_count?type={type_n[0]}&interval_id=100%3A90'
        total = requests.get(url=json_url, headers=headers).json()['total']
        # 获取电影
        movie_get(type_n, total)


def movie_get(type_n, total):
    # 处理分页
    pages = math.ceil(total/20)
    # 请求每个异步列表获取全部电影介绍
    for i in range(pages):
        m_url = f'https://movie.douban.com/j/chart/top_list?type={type_n[0]}&interval_id=100%3A90&action=&start={i*20}&limit=20'
        # 请求之后得到列表，再从列表中得到数据
        m_list = requests.get(url=m_url, headers=headers).json()
        # print(m_list)
        for m in m_list:
            # print(m)
            global num
            num += 1
            movie_detail = {}
            movie_detail['电影名'] = m['title']
            movie_detail['类型'] = m['types']
            movie_detail['排名'] = m['rank']
            movie_detail['评分'] = m['score']
            movie_detail['上映地区'] = m['regions']
            movie_detail['演员'] = m['actors']
            # 保存数据
            save(movie_detail)
            print('第' + str(num) + '部电影保存成功')


def save(m):
    with open('豆瓣电影.txt', 'a', encoding='utf8') as f:
        f.write(str(m)+'\n')


if __name__ == '__main__':
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
    }
    base_url = 'https://movie.douban.com/chart'
    num = 0
    page_get()



